package Retrieval;

use strict;

use Clair::Utils::Tf;
use API::Document;

sub execute_query {
  my @searchTerms = @_;
  my $tf = Clair::Utils::Tf->new(rootdir => "produced", corpusname => "cranfieldcorpus", stemmed => 1);
  my %score_hash = ();  # [OUTPUT #1] key: doc_id, value: score
  my %loc_hash = ();    # [OUTPUT #2] key: doc_id, value: location of first result
  my @negation;         # the array storing negation words and phrases
  my $negationOnly = 1;

  foreach my $term (@searchTerms) {
    if ($term =~ s/!//g) {
      push(@negation, $term);
    } else {
      $negationOnly = 0;
      my @words = split(/ /, $term);
      my $numWords = @words;
      # [url_pos_hash] key: document URL (file name with path)
      #                value: a ref to an array of all positions 
      #                       at which the phrase occurs in file "key" 
      my $url_pos_hash = $tf->getDocsWithPhrase(@words);

      while ( my ($docId, $ref_hash) = each %$url_pos_hash ) {
        my $score = keys(%$ref_hash);
        $score *= $numWords;
        $docId =~ s/.*\/([0-9]+)\.txt/$1/g;
        if (!exists $loc_hash{$docId}){
          my ($position, $storedVal) = each %$ref_hash;  #just need the first position
          $loc_hash{$docId} = $position;
        }

        if (exists $score_hash{$docId}){
          $score_hash{$docId} += $score;
        } else {
          $score_hash{$docId} = $score;
        }
      }
    }
  }

  if ($negationOnly == 1) {
    %score_hash = getAllDocKeys($tf);
    foreach my $key (keys %score_hash){
      $loc_hash{$key} = 0;
    }
  }

  if ((scalar @negation) > 0){
    # the list of docs that contain negated qurey terms
    my @negatedDocs = negationResults(\@negation, $tf);
    foreach my $removeDoc (@negatedDocs){
      if (exists $score_hash{$removeDoc}){
        delete $score_hash{$removeDoc};
      }
    }
  }

  return \%score_hash, \%loc_hash;
}


sub find_similar_context {
  my $word = shift;
  chomp $word;
  my $tf = Clair::Utils::Tf->new(rootdir => "produced", corpusname => "cranfieldcorpus", stemmed => 1);
  my $doc_path = 'txt_corpus';

  my @docs = $tf->getDocs($word);
  my %contexts = ();

  foreach my $docId (@docs) {
    $docId =~ s/.*\/([0-9]+)\.txt/$1/g;
    # open docId.txt
    my $text = Document::slurp_file "$doc_path/$docId.txt";

    # find all contexts in docId.txt
    while ( $text =~ /(\w+) $word (\w+)/g ) { 
      my $context = "$1 $2";

      # add the $context to %contexts hash
      if (defined $contexts{$context}) {
        $contexts{$context}++;
      } else {
        $contexts{$context} = 1;
      }
    }
  }

  my %result = ();
  foreach my $k (keys %contexts) {
    my @con = split / /, $k; 
    system("grep -roh \"$con[0] \\w\\+ $con[1]\" $doc_path/ > temp.txt");
    open (my $FH, "temp.txt");
    while (my $line = <$FH>) {
      $line =~ s/\w+ (\w+) \w+/$1/;
      chomp $line;
      if(defined $result{$line}) { $result{$line} += $contexts{$k}; }
      else { $result{$line} = $contexts{$k}; }
    }
  }
  system("rm -f temp.txt");

  delete $result{$word};
  if (defined $result{"the"}) { delete $result{"the"}; }
  if (defined $result{"a"}) { delete $result{"a"}; }
  if (defined $result{"an"}) { delete $result{"an"}; }
  foreach my $k1 (sort {$result{$b} <=> $result{$a}} keys %result) {
    last if ($result{$k1} < 2);
    print "$k1, $result{$k1}\n";
  }
}




sub getAllDocKeys {
  my %url_docid = ();
  my %allDocKeys = ();
  my $DOCID_DBM = 'produced/corpus-data/cranfieldcorpus/cranfieldcorpus-url-to-docid';

  dbmopen %url_docid, $DOCID_DBM, 0 or die "Cannot open $DOCID_DBM";
  while (my ($k, $v) = each %url_docid) {
    $k =~ s/.*\/([0-9]+)\.txt/$1/g;
    $allDocKeys{$k} = 0;
  }
  dbmclose %url_docid;

  return %allDocKeys;
}

sub negationResults {
  my $searchTerms = shift;
  my $tf = shift;
  my @doc_list = ();

  foreach my $term (@$searchTerms) {
    my @words = split(/ /, $term);
    my $url_pos_hash = $tf->getDocsWithPhrase(@words);

    foreach my $docId (keys %$url_pos_hash) {
      $docId =~ s/.*\/([0-9]+)\.txt/$1/g;
      push @doc_list, $docId;
    }
  }
  return @doc_list;
}

sub find_docs {
  my $term = shift;
  chomp $term;

  my @searchTerm =(); 
  # extract the phrase if there is one
  if ($term =~ s/\"(.+?)\"/$1/){
    @searchTerm = split / /, $term;
  } else {
  # extract the word if there is no phrase
    push(@searchTerm, $term);
  }

  my $tf = Clair::Utils::Tf->new(rootdir => "produced", corpusname => "cranfieldcorpus", stemmed => 1);
  my $url_pos_hash = $tf->getDocsWithPhrase(@searchTerm);
  if(%$url_pos_hash) {
    my $num = keys(%$url_pos_hash);
    print "There are $num documents containing \"@searchTerm\"\n";
  } else {
    print "Could not find \"@searchTerm\" in any doc\n";
  }
}

sub find_freq {
  my $term = shift;
  chomp $term;

  my @searchTerm =(); 
  # extract the phrase if there is one
  if ($term =~ s/\"(.+?)\"/$1/){
    @searchTerm = split / /, $term;
  } else {
  # extract the word if there is no phrase
    push(@searchTerm, $term);
  }

  my $tf = Clair::Utils::Tf->new(rootdir => "produced", corpusname => "cranfieldcorpus", stemmed => 1);
  my $count = $tf->getPhraseFreq(@searchTerm);
  print "\"@searchTerm\" appeared $count times\n";
}

1;

